#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@File    :   renren_spider.py
@Time    :   2019/08/01 20:54:48
@Author  :   JIAWEI GAO
@Contact :   jiawei.goah@gmail.com
@TODO    :   Have a good day!
'''
import random
import re
import time

import pandas as pd
import requests
import simplejson as json
from bs4 import BeautifulSoup
from pandas import DataFrame, Series


class renren_handler(object):

    def __init__(self, filename):
        #filename = input('Please input your file name.')
        user_agents = [
            'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
            'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
            'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
            'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0',
            'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
            'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',
            'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)',
            'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)',
            'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)'
        ]
        user_agent = random.choice(user_agents)
        #query_url = url.format(word)
        #print(query_url)
        self.headers = {
        'user-agent': user_agent
        }

        self.times = [1.234, 4.343, 2.435, 3.34, 2.34, 3.342]

        word_list = DataFrame()
        """for i in range(10,16):
            wordlisti = DataFrame()
            print('wl{}.xlsx'.format(i))
            wordlisti = pd.read_excel('wl{}.xlsx'.format(i), header = None)
            word_list = word_list.append(wordlisti, ignore_index=True)"""
        wordlist_plus = pd.read_excel('{}.xlsx'.format(filename), header = None)
        self.word_list = word_list.append(wordlist_plus, ignore_index=True)
    
    def get_urls(self, word):
        url = 'http://www.91dict.com/words?w={}'.format(word)
        wb_data = requests.get(url, headers = self.headers)
        soup = BeautifulSoup(wb_data.text, 'lxml')

        image_url = soup.select('#flexslider_2 > ul.slides.clearfix > li:nth-child(1) > div.imgMainbox > img')[0]['src']
        captian_eng = soup.select('#flexslider_2 > ul.slides.clearfix > li:nth-child(1) > div.imgMainbox > div > div.mBottom')[0].get_text()
        captian_chs = soup.select("#flexslider_2 > ul.slides.clearfix > li:nth-child(1) > div.imgMainbox > div > div.mFoot")[0].get_text()
        audio_url = soup.select('#flexslider_2 > ul.slides.clearfix > li:nth-child(1) > div.imgMainbox > div > div.mTop > audio')[0]['src']
        return [image_url, audio_url, captian_chs, captian_eng]

    def save_img(self, image_url, title):
        with open('{}.jpg'.format(title), 'wb') as f:
            img = requests.get(image_url, headers = self.headers).content
            f.write(img)

    def save_audio(self, audio_url, title):
        with open('{}.mp3'.format(title), 'wb') as f:
            img = requests.get(audio_url, headers = self.headers).content
            f.write(img)

    def get_data(self):
        dataframe = list()
        j = 1
        #网页词汇用下面这个for循环
        #for word in word_list:
        #词汇列表用下面这个for循环
        for word in self.word_list[0]:
            print(j)
            j = j + 1
            print(word)
            
            time_step = random.choice(self.times)
            time.sleep(time_step)
            try:
                info_list = self.get_urls(word)
                print(info_list[3])
                self.save_img(info_list[0], word)
                self.save_audio(info_list[1], word)
                info_list.append(word)
                dataframe.append(info_list)
            except IndexError:
                pass
            
        dataframe = DataFrame(dataframe)
        func = lambda x:x.split('\n')[1]
        dataframe[5] = dataframe[2].apply(func)
        dataframe[6] = dataframe[3].apply(func)
        del dataframe[0]
        del dataframe[1]
        del dataframe[2]
        del dataframe[3]
        func_sound = lambda x:'[sound:{}.mp3]'.format(x)
        func_img = lambda x:'<img src="{}.jpg">'.format(x)
        dataframe[7] = dataframe[4].apply(func_sound)
        dataframe[8] = dataframe[4].apply(func_img)
        dataframe.head()
        dataframe.to_csv('word_list_finish.csv', header = None, index = False)
